<?php

class cronProcessqueueTask extends sfBaseTask
{
  protected function configure()
  {
    // // add your own arguments here
    // $this->addArguments(array(
    //   new sfCommandArgument('my_arg', sfCommandArgument::REQUIRED, 'My argument'),
    // ));

    $this->addOptions(array(
      new sfCommandOption('application', null, sfCommandOption::PARAMETER_REQUIRED, 'The application name'),
      new sfCommandOption('env', null, sfCommandOption::PARAMETER_REQUIRED, 'The environment', 'dev'),
      new sfCommandOption('connection', null, sfCommandOption::PARAMETER_REQUIRED, 'The connection name', 'doctrine'),
      // add your own options here
    ));

    $this->namespace        = 'cron';
    $this->name             = 'process-queue';
    $this->briefDescription = 'Processes Queue and crawls';
    $this->detailedDescription = <<<EOF
The [cron:process-queue|INFO] task does things.
Call it with:

  [php symfony cron:process-queue|INFO]
EOF;
  }

  protected function execute($arguments = array(), $options = array())
  {
    // initialize the database connection
    $databaseManager = new sfDatabaseManager($this->configuration);
    $connection = $databaseManager->getDatabase($options['connection'])->getConnection();

    //Load single url from queue
    $queue = Doctrine_Core::getTable('Queue')->createQuery('q')->where('q.in_process = ?', 0)->limit(1)->execute();

    if(empty($queue[0]))
        die("The Queue is empty");
    
    $url = $queue[0]->getUrl();

    $base_url = $this->get_base_url($url);

    echo "Processing url: {$url}\n";
    //$queue[0]->setInProcess(1)->save();

    // Check if page is already crawled
    $num_pages = Doctrine_Core::getTable('Page')->findByUrl($url)->count();
    if($num_pages == 0)
    { // Creating new Page object
        $this->page = new Page();
        $this->page->setUrl($url)->save();
    }
    else
    {
      // Getting existing page object
      //$this->page = Doctrine_Core::getTable('Page')->findOneByUrl($base_url);
      
      $queue[0]->delete();
      echo "Page already crawled, skipping..."; return false;

    }

    $page_id = $this->page->getId();

    $b = new sfWebBrowser();

    $dom = $b
        ->get($url)
        ->getResponseDom();

    if($dom)
    {
      $links = $dom
        ->getElementsByTagName('a');
      $h1s = $dom
        ->getElementsByTagName('h1');
      $h2s = $dom
        ->getElementsByTagName('h2');
      $h3s = $dom
        ->getElementsByTagName('h3');
      $metas = $dom
        ->getElementsByTagName('meta');

    }
 else {
      echo "Unable to load DOM on a link: ".$url.'. Exiting...';
      $queue[0]->delete();
      return false;
    }

    foreach($links as $link)
    {
        $href = $link->getAttribute('href');
        $title = $link->getAttribute('title');
        $rel = $link->getAttribute('rel');

        if(empty($title))   $title = $link->nodeValue;

        if(substr($href, 0,1) == './') $href = $base_url.substr ($href, 1);
        if(substr($href, 0,1) == '/') $href = $base_url.$href;

        if($href=='#' && $rel != '') $href= 'mailto:'.$this->convert_from_ascii_hash($rel);

	if(substr($href, 0,7) != 'http://' && substr($href, 0,6) != 'mailto') $href = $base_url.'/'.$href;


        $href = str_replace('//', '/', $href);
        $href = str_replace('http:/', 'http://', $href);

        if(substr($href, 0,1) != '#')
        { // Save link to DB
            echo $href."\n";
            $this->save_link($href, $title, $page_id);
            $this->add_to_queue($href);
        }
    }
    // Delete this page from queue
    $queue[0]->delete();
  }

  private function save_email($email, $page_id)
  {
      $cnt = Doctrine_Core::getTable('Email')
                ->createQuery('m')
                ->where('m.url = ?', $email)
                ->andWhere('page_id = ?', $page_id)
                ->count();

      if($cnt==0)
      {
          $email_obj = new Email();
          $email_obj->setUrl($email)->setPage($this->page)->save();
          $email_obj->free();
      }
  }

  private function save_link($url, $title, $page_id)
  {
      if($this->validate_link($url)==false)
        return false;
      $cnt = Doctrine_Core::getTable('Link') //->findOneByUrlAndPageId()->count()
                ->createQuery('l')
                ->where('url = ?', $url)
                ->andWhere('page_id = ?', $page_id)
                ->count();
      if($cnt==0)
      {
          $pos = strpos($url, 'mailto:');
          if($pos === false)
          {
              $link = new Link();
              $link->setUrl($url)->setPage($this->page)->setTitle($title)->save();
              $link->free();
          }
          else {
              
              $mail = substr($url, 7);
              $this->save_email($mail, $page_id);
          }
      }

  }
  private function validate_link($url)
  {
    $pos = strpos($url, 'javascript:');
    if($pos === false)
        return true;
    else
        return false;

  }

  private function add_to_queue($url)
  {
      $url = $this->normalize_url($url);
      $cnt = Doctrine_Core::getTable('Queue')->findByUrl($url)->count();
      $cnt2 = PageTable::getInstance()->findByUrl($url)->count();

      // Check if this link has already been indexed
      if(($cnt+$cnt2)==0)
      {
        $queue = new Queue();
        $queue->setUrl($url)->save();
        $queue->free();
      }

  }

  private function get_base_url($url)
  {
    $uc = parse_url($url);
    return $uc['scheme'].'://'.$uc['host'].'/';

  }

  private function normalize_url($url)
  {
      if(substr($url, -1) == '/')
        return substr($url, 0, -1);
      else
        return $url;
  }

  function convert_from_ascii_hash($ah) {

    $str = "";

    for($i=0; $i<(strlen($ah)-1); $i=$i+3)
    {
      $charval = substr($ah, $i, 3);
      $str .= chr($charval);
    }

    return $str;

}
}
